# -*- coding: utf-8 -*-
'''
Created on Aug 4, 2012

@author: LONG HOANG GIANG
'''


from urlparse import urljoin
import datetime
import re
import sys
import os
import traceback
sys.path.append(os.path.expanduser('/home5/vietcntt/longhoanggiang/python'))
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib2 import commonlib, html2text
from Model.storymodelv3 import StoryModel

def cleanElementWithAttrib(element, tag='', attrib=[()]):
    '''Ham loai bo child node cua element
       - loai theo tag voi attrib
       - loai tat ca phu hop voi attrib
    '''
    try:
        if element == None: return ''
        if type(element).__name__ == 'list': element = element[0]
        if tag == '': tag = None
        for elem in element.iterdescendants(tag=tag):
            f = True
            for (i, v) in attrib:
                if elem.attrib.get(i, '') != v: f = False; break
            if f:
                pElem = elem.getparent()
                if pElem != None: pElem.remove(elem)  
    except:
        traceback.print_exc()

def getListStory():
    print '>start get list story'
    url = 'http://www.truyencuoivova.net/category/truy%E1%BB%87n-c%C6%B0%E1%BB%9Di-vova-full/'
#    url = 'http://www.truyencuoivova.net/category/truy%E1%BB%87n-c%C6%B0%E1%BB%9Di-dan-gian/'
#    url = 'http://www.truyencuoivova.net/category/truyen-cuoi-tieu-lam/'
    data = []
    while True:
        tree = commonlib.loadweb(url).build_tree()
        for item in tree.xpath("//h2[contains(@id, 'post-') and @class='title']/a"):
            title = commonlib.stringify(item)
            title = re.sub(r"Truyện cười Vova : ", "", title)
            link = urljoin(url, item.get('href'))
            data.append({'title': title, 'url': link})
        nextPageNode = tree.xpath("//div[@id='wp_page_numbers']/ul/li[@class='active_page']/following-sibling::*[1]/a")
        if len(nextPageNode) == 0:
            print '>> Trang cuoi'
            break
        url = urljoin(url, nextPageNode[0].get('href'))
    return data

def getContent(item):
    
    url = item['url']
    print '------- get content {0}: {1}'.format(item['title'], url)
    tree = commonlib.loadweb(url).build_tree()
    contentNode = tree.xpath("//div[@class='entry']")
    for child in contentNode[0].xpath(".//div[contains(@style, 'display')]"):
        for inext in child.xpath("./following-sibling::*"):
            inext.getparent().remove(inext)
        child.getparent().remove(child)
    titleNode = contentNode[0].xpath(".//*[contains(., 'Truyện cười')]".decode('utf-8'))
    if len(titleNode) > 0:
        titleNode = titleNode[0]
        titleNode.getparent().remove(titleNode)
    html = html2text.html2text(commonlib.Etree.tostring(contentNode)).strip()
    html = re.sub(r"\n", "<br />", html)
    html = '''<h4>{0}</h4><br /><br />{1}'''.format(item['title'], html)
    return html

def createTable(c):
    c.execute("DROP TABLE IF EXISTS tblChapters")
    c.execute("DROP TABLE IF EXISTS tblDetail")
    c.execute("CREATE TABLE tblChapters(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT)")
    c.execute("CREATE INDEX chapter_idx ON tblChapters(id)")
    c.execute("CREATE TABLE tblDetail(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `chapterId` INTEGER, `detail` TEXT)")
    c.execute("CREATE INDEX detail_idx ON tblDetail(chapterId)")

#def updateDb():
#    conn = db.connect('cuoidangian')
#    c = conn.cursor()
#    c.execute("SELECT * FROM tblChapters")
#    titles = c.fetchall()
#    
#    for title in titles:
#        a = re.sub(r"Truyện cười dân gian :".decode('utf-8'), "", title[1]).strip()
#        a = re.sub(r"Truyện cười ngắn :".decode('utf-8'), "", a).strip()
#        a = re.sub(r"Truyện cười tiếu lâm :".decode('utf-8'), "", a).strip()
#        c.execute('''UPDATE tblChapters SET title='{0}' WHERE id={1}'''.format(a, title[0]))
#    c.close()
#    conn.commit()
#    conn.close()
    
def normalizeTitle(string):
    output = re.sub(r"Truyện cười dân gian :", "", string)
    output = re.sub(r"Truyện cười ngắn :", "", output)
    output = re.sub(r"Truyện cười tiếu lâm :", "", output)
    output = re.sub(r"Truyện cười nghề nghiệp :", "", output)
    output = re.sub(r"Truyện cười 18\+ :", "", output)
    return commonlib.toUpper(output.strip())

def process():
    links = getListStory()
    smodel = StoryModel(DATABASE)
    smodel.open()
    for item in links:
        try:
            title = normalizeTitle(item['title'])
            detail = getContent(item)
            if detail != '' and title != '':
                smodel.add_story(title, detail, 0)
        except:
            pass
    smodel.close()
            
if __name__ == '__main__':
    
    DATABASE = '/longhoanggiang/database/vova'
    
    process()
#    print getContent({'title': 'haha', 'url': 'http://www.truyencuoivova.net/truy%E1%BB%87n-c%C6%B0%E1%BB%9Di-vova-gi%E1%BB%9D-h%E1%BB%8Dc-sinh-v%E1%BA%ADt/'})
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
    
    
    